Keras Multi-layer Perceptron
Tensorflow Premade Estimators
Tensorflow Image and Text Classification
import numpy as np
import pandas as pd
from sklearn import datasets
data = datasets.load_breast_cancer()
Data = pd.DataFrame(data['data'], columns = [x.title() for x in data['feature_names']])
Labels_dict = dict(zip(list(np.sort(np.unique(data['target'].tolist()))),
list([x.title() for x in data['target_names']])))
Target = 'Diagnosis'
Data[Target] = data['target']
# Data['Diagnosis'] = data['target'].replace(Labels_dict)
display(Data)
print(data['DESCR'])
Moreover, high variance for some features can hurt our modeling process. For this reason, we would like to standardize features by removing the mean and scaling to unit variance.
from sklearn import preprocessing
X = Data.drop(columns = [Target])
y = Data[Target]
Temp = X.var().to_frame(name= 'Variance (Origial)').round(4)
scaler = preprocessing.StandardScaler()
X_std = scaler.fit_transform(X)
X_std = pd.DataFrame(data = X_std, columns = X.columns)
Temp = Temp.join(X_std.var().to_frame(name= 'Variance (Normalized)').round(4))
display(Temp.style.background_gradient(cmap='Reds', subset = 'Variance (Origial)')\
.background_gradient(cmap='Greens', subset = 'Variance (Normalized)'))
del Temp
import plotly.express as px
from HD_DeepLearning import DatasetTargetDist
Pull = [0 for x in range((len(Labels_dict)-1))]
Pull.append(.05)
PD = dict(PieColors = ['SeaGreen','FireBrick'], TableColors = ['Navy','White'], hole = .4,
column_widths=[0.6, 0.4], textfont = 14, height = 400, tablecolumnwidth = [0.25, 0.15, 0.15],
pull = Pull, legend_title = Target, title_x = 0.5, title_y = .9, pie_legend = [0.01, 0.01])
del Pull
DatasetTargetDist(Data, Target, Labels_dict, PD, orientation= 'columns')
StratifiedKFold is a variation of k-fold which returns stratified folds: each set contains approximately the same percentage of samples of each target class as the complete set.
from sklearn.model_selection import StratifiedShuffleSplit
def HD_StratifiedShuffleSplit(X, y, Test_Size = 0.3):
sss = StratifiedShuffleSplit(n_splits=1, test_size=Test_Size, random_state=42)
_ = sss.get_n_splits(X, y)
for train_index, test_index in sss.split(X, y):
# X
if isinstance(X, pd.DataFrame):
X_train, X_test = X.loc[train_index], X.loc[test_index]
else:
X_train, X_test = X[train_index], X[test_index]
# y
if isinstance(y, pd.Series):
y_train, y_test = y[train_index], y[test_index]
else:
y_train, y_test = y[train_index], y[test_index]
del sss
return X_train, y_train, X_test, y_test
X_train, y_train, X_test, y_test = HD_StratifiedShuffleSplit(X_std, y)
from HD_DeepLearning import Train_Test_Dist
PD.update(dict(column_widths=[0.3, 0.3, 0.3], tablecolumnwidth = [0.2, 0.4], height = 550, legend_title = Target))
Train_Test_Dist(X_train, y_train, X_test, y_test, PD, Labels_dict)
#
import tensorflow as tf
# y_train = tf.keras.utils.to_categorical(y_train, num_classes=len(Labels_dict))
# y_test = tf.keras.utils.to_categorical(y_test, num_classes=len(Labels_dict))
Create the feature columns, using the original numeric columns as is and one-hot-encoding categorical variables.
def Feat_Columns(Inp, Numeric = False, disp_dtype = False):
'''
Feature Columns function
Input: Dataset
Output: Tensorflow Feature Column List
'''
if not Numeric:
Numeric = ['int64', 'int32', 'float64', 'float32']
Temp = Inp.dtypes.reset_index(drop = False)
Temp.columns = ['Features', 'Data Type']
Temp['Data Type'] = Temp['Data Type'].astype(str)
# Numeric_Columns
Numeric_Columns = Temp.loc[Temp['Data Type'].isin(Numeric), 'Features'].tolist()
# Categorical_Columns
# Categorical_Columns = Temp.loc[(~Temp['Data Type'].isin(Numeric)), 'Features'].tolist()
Categorical_Columns = Temp.loc[Temp['Data Type'] == 'object','Features'].tolist()
if disp_dtype:
display(pd.DataFrame({'Numeric Columns': [', '.join(Numeric_Columns)],
'Categorical Columns': [', '.join(Categorical_Columns)]}, index = ['Columns']).T.style)
# Feature Columns
feature_columns = []
if len(Categorical_Columns)>0:
for feature_name in Categorical_Columns:
vocabulary = Inp[feature_name].unique()
feature_columns.append(tf.feature_column.indicator_column(\
tf.feature_column.categorical_column_with_vocabulary_list(feature_name, vocabulary)))
if len(Numeric_Columns)>0:
for feature_name in Numeric_Columns:
feature_columns.append(tf.feature_column.numeric_column(feature_name))
return feature_columns
def make_input_fn(X, y, inmemory_train = False, n_epochs= None, shuffle=True, batch_size = 256):
# Not In memory Training
if not inmemory_train:
def input_fn():
dataset = tf.data.Dataset.from_tensor_slices((X.to_dict(orient='list'), y))
if shuffle:
dataset = dataset.shuffle(1000)
dataset = (dataset.repeat(n_epochs).batch(batch_size))
return dataset
# In memory Training
if inmemory_train:
y = np.expand_dims(y, axis=1)
def input_fn():
return dict(X), y
# End
return input_fn
my_feature_columns = Feat_Columns(X)
# Training and evaluation input functions.
train_input_fn = make_input_fn(X_train, y_train)
eval_input_fn = make_input_fn(X_test, y_test, shuffle=False, n_epochs=1)
from IPython.display import clear_output
from timeit import default_timer as timer
# Classifier
tf.keras.backend.clear_session()
IT = int(1e3)
params = {'n_trees': 50, 'max_depth': 3, 'n_batches_per_layer': 1, 'center_bias': True}
classifier = tf.estimator.BoostedTreesClassifier(my_feature_columns, **params)
# Train model.
start = timer()
classifier.train(train_input_fn, max_steps = IT)
CPU_Time = timer() - start
# Evaluation.
results = classifier.evaluate(eval_input_fn)
clear_output()
results['CPU Time'] = CPU_Time
display(pd.DataFrame(results, index = ['']).round(4))
from HD_DeepLearning import ROC_Curve
# converting y_test to categorical
y_test_cat = tf.keras.utils.to_categorical(y_test, num_classes = len(Labels_dict), dtype='float32')
pred_dicts = list(classifier.predict(input_fn=eval_input_fn))
clear_output()
probs = np.array([pred['probabilities'] for pred in pred_dicts])
ROC_Curve(y_test_cat, probs, n_classes = len(Labels_dict), FS = 8)
The confusion matrix allows for visualization of the performance of an algorithm. Note that due to the size of data, here we don't provide a Cross-validation evaluation. In general, this type of evaluation is preferred.
def Confusion_Mat(CM_Train, CM_Test, PD, n_splits = 10):
if n_splits == None:
Titles = ['Train Set', 'Test Set']
else:
Titles = ['Train Set (CV = % i)' % n_splits, 'Test Set (CV = % i)' % n_splits]
CM = [CM_Train, CM_Test]
Cmap = ['Greens', 'YlGn','Blues', 'PuBu']
for i in range(2):
fig, ax = plt.subplots(1, 2, figsize= PD['FS'])
fig.suptitle(Titles[i], weight = 'bold', fontsize = 16)
_ = sns.heatmap(CM[i], annot=True, annot_kws={"size": PD['annot_kws']}, cmap=Cmap[2*i], ax = ax[0],
linewidths = 0.2, cbar_kws={"shrink": PD['shrink']})
_ = ax[0].set_title('Confusion Matrix');
Temp = np.round(CM[i].astype('float') / CM[i].sum(axis=1)[:, np.newaxis], 2)
_ = sns.heatmap(Temp,
annot=True, annot_kws={"size": PD['annot_kws']}, cmap=Cmap[2*i+1], ax = ax[1],
linewidths = 0.4, vmin=0, vmax=1, cbar_kws={"shrink": PD['shrink']})
_ = ax[1].set_title('Normalized Confusion Matrix');
for a in ax:
_ = a.set_xlabel('Predicted labels')
_ = a.set_ylabel('True labels');
_ = a.xaxis.set_ticklabels(PD['Labels'])
_ = a.yaxis.set_ticklabels(PD['Labels'])
_ = a.set_aspect(1)
An alternative way to train a model with boosting performance is using the train_in_memory feature. However, if there is no issue with performance or long training time is not a concern, training without this feature is recommended [2]. Furthermore, our observations have shown that using train_in_memory not always increases the performance of the training.
in_memory_params = dict(params)
in_memory_params['n_batches_per_layer'] = 1
# In-memory input_fn does not use batching.
train_input_fn = make_input_fn(X_train, y_train, inmemory_train = True)
# Classifier
tf.keras.backend.clear_session()
classifier = tf.estimator.BoostedTreesClassifier(my_feature_columns, train_in_memory=True, **in_memory_params)
# Train model.
start = timer()
classifier.train(train_input_fn, max_steps = IT)
CPU_Time = timer() - start
# Evaluation.
results = classifier.evaluate(eval_input_fn)
clear_output()
results['CPU Time'] = CPU_Time
display(pd.DataFrame(results, index = ['']).round(4))
# converting y_test to categorical
y_test_cat = tf.keras.utils.to_categorical(y_test, num_classes = len(Labels_dict), dtype='float32')
pred_dicts = list(classifier.predict(input_fn=eval_input_fn))
clear_output()
probs = np.array([pred['probabilities'] for pred in pred_dicts])
ROC_Curve(y_test_cat, probs, n_classes = len(Labels_dict), FS = 8)
We can investigate the feature importance of an artificial classification task. This is similar to that of scikit-learn and has been outlined in [6].
pred_dicts = list(classifier.experimental_predict_with_explanations(eval_input_fn))
clear_output()
# Create DFC Pandas dataframe.
labels = y_test
probs = pd.Series([pred['probabilities'][1] for pred in pred_dicts])
df_dfc = pd.DataFrame([pred['dfc'] for pred in pred_dicts])
df_dfc.columns = [x.replace('_',' ') for x in df_dfc.columns]
display(df_dfc.describe().T.style.background_gradient(subset= ['mean'], cmap='RdYlGn')\
.background_gradient(subset= ['std'], cmap='RdYlGn')\
.background_gradient(subset= ['min'], cmap='hot')\
.background_gradient(subset= ['max'], cmap='winter')
.format(precision=4).format({'count': "{:.0f}"}))
A nice property of DFCs is that the sum of the contributions + the bias is equal to the prediction for a given example.
# Sum of DFCs + bias == probabality.
bias = pred_dicts[0]['bias']
dfc_prob = df_dfc.sum(axis=1) + bias
np.testing.assert_almost_equal(dfc_prob.values, probs.values)
Plot DFCs for an individual patient which is color-coded based on the contributions' directionality and add the feature values on the figure.
import matplotlib.pyplot as plt
from matplotlib.font_manager import FontProperties
def _add_feature_values(feature_values, ax, colors):
"""Display feature's values on left of plot."""
x_coord = ax.get_xlim()[0]
OFFSET = 0.15
for y_coord, (feat_name, feat_val) in enumerate(feature_values.items()):
t = plt.text(x_coord, y_coord - OFFSET, '{}'.format(feat_val), size=12)
t.set_bbox(dict(facecolor= colors[y_coord], alpha=0.25))
font = FontProperties()
# font.set_weight('bold')
t = plt.text(x_coord, y_coord + 1 - OFFSET, 'Feature\nValue', fontproperties=font, size=13)
def _xLims(ax):
Temp = np.linspace(-1,1,21, endpoint=True)
Temp = np.round(Temp,1)
xlims = ax.get_xlim()
for l, r in list(zip(Temp[:-1],Temp[1:])):
if l<= xlims[0] < r:
Left = l
if l<= xlims[1] < r:
Right = r
return [Left, Right]
def Plot_Example(example, TOP_N = 10, Pos_Color = 'LimeGreen', Neg_Color = 'OrangeRed', Maps = None, FS = (13, 7)):
# Sorting by absolute value
sorted_ix = example.abs().sort_values()[-TOP_N:].index
example = example[sorted_ix]
fig, ax = plt.subplots(1, 1, figsize= FS)
Temp = example.to_frame('Value').sort_index(ascending= False)
Temp0 = Temp.copy(); Temp0[Temp0 < 0] = np.nan
_ = Temp0.plot(kind='barh', color= Pos_Color, edgecolor = 'white', hatch = '///', legend=None, alpha=0.75, ax = ax)
Temp0 = Temp.copy(); Temp0[Temp0 >= 0] = np.nan
_ = Temp0.plot(kind='barh', color= Neg_Color, edgecolor = 'white', hatch = '///', legend=None, alpha=0.75, ax = ax)
_ = Temp.plot(kind='barh', color='None', edgecolor = 'Black', legend=None, alpha=1, lw=1.2, ax = ax)
del Temp, Temp0
_ = ax.grid(False, axis='y')
# x axis Limits
_ = ax.set_xlim(_xLims(ax))
# Add feature values.
Temp = X_test.copy()
Temp.columns = [x.replace('_',' ') for x in Temp.columns]
if not Maps == None:
for c in Maps.keys():
Temp[c] = Temp[c].map(Maps[c])
colors = example.map(lambda x: Pos_Color if x >= 0 else Neg_Color).tolist()
_add_feature_values(Temp.iloc[ID][sorted_ix].round(4), ax, colors)
return ax
ID = 61
Tops = X_train.shape[1]
ax = Plot_Example(df_dfc.iloc[ID], TOP_N = Tops, FS = (13, 16))
_ = ax.set_title('Feature contributions for example patient {} from the Test set\n Pred: {:1.2f}; Label: {}'
.format(ID, probs[ID], labels.iloc[ID]))
_ = ax.set_xlabel('Contribution to Predicted Probability', size=14)
Detrano, R., Janosi, A., Steinbrunn, W., Pfisterer, M., Schmid, J.J., Sandhu, S., Guppy, K.H., Lee, S. and Froelicher, V., 1989. International application of a new probability algorithm for the diagnosis of coronary artery disease. The American journal of cardiology, 64(5), pp.304-310.
Aha, D. and Kibler, D., 1988. Instance-based prediction of heart-disease presence with the Cleveland database. University of California, 3(1), pp.3-2.
Gennari, J.H., Langley, P. and Fisher, D., 1989. Models of incremental concept formation. Artificial intelligence, 40(1-3), pp.11-61.